{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import gc\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('max_columns', 100)\n",
    "pd.set_option('max_rows', 100)\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import GroupKFold, KFold\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('raw_data/train.csv')\n",
    "train = train.sort_values(by=['QUEUE_ID', 'DOTTING_TIME']).reset_index(drop=True)\n",
    "\n",
    "test = pd.read_csv('raw_data/evaluation_public.csv')\n",
    "test = test.sort_values(by=['ID', 'DOTTING_TIME']).reset_index(drop=True)\n",
    "\n",
    "sub_sample = pd.read_csv('raw_data/submit_example.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>QUEUE_ID</th>\n",
       "      <th>CU</th>\n",
       "      <th>STATUS</th>\n",
       "      <th>QUEUE_TYPE</th>\n",
       "      <th>PLATFORM</th>\n",
       "      <th>CPU_USAGE</th>\n",
       "      <th>MEM_USAGE</th>\n",
       "      <th>LAUNCHING_JOB_NUMS</th>\n",
       "      <th>RUNNING_JOB_NUMS</th>\n",
       "      <th>SUCCEED_JOB_NUMS</th>\n",
       "      <th>CANCELLED_JOB_NUMS</th>\n",
       "      <th>FAILED_JOB_NUMS</th>\n",
       "      <th>DOTTING_TIME</th>\n",
       "      <th>RESOURCE_TYPE</th>\n",
       "      <th>DISK_USAGE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>3</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1590683100000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1590683400000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>7</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1590683700000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>4</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1590684000000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>5</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1590684120000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>3</td>\n",
       "      <td>55</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1590684420000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1590684720000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1590685020000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>5</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1590685320000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>6</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1590685620000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   QUEUE_ID  CU     STATUS QUEUE_TYPE PLATFORM  CPU_USAGE  MEM_USAGE  \\\n",
       "0         2  16  available        sql   x86_64          3         54   \n",
       "1         2  16  available        sql   x86_64          2         54   \n",
       "2         2  16  available        sql   x86_64          7         54   \n",
       "3         2  16  available        sql   x86_64          4         54   \n",
       "4         2  16  available        sql   x86_64          5         54   \n",
       "5         2  16  available        sql   x86_64          3         55   \n",
       "6         2  16  available        sql   x86_64          2         54   \n",
       "7         2  16  available        sql   x86_64          2         54   \n",
       "8         2  16  available        sql   x86_64          5         54   \n",
       "9         2  16  available        sql   x86_64          6         54   \n",
       "\n",
       "   LAUNCHING_JOB_NUMS  RUNNING_JOB_NUMS  SUCCEED_JOB_NUMS  CANCELLED_JOB_NUMS  \\\n",
       "0                   0                 0                 0                   0   \n",
       "1                   0                 0                 0                   0   \n",
       "2                   0                 0                 0                   0   \n",
       "3                   0                 0                 0                   0   \n",
       "4                   0                 0                 0                   0   \n",
       "5                   0                 0                 0                   0   \n",
       "6                   0                 0                 0                   0   \n",
       "7                   0                 0                 0                   0   \n",
       "8                   0                 0                 0                   0   \n",
       "9                   0                 0                 0                   0   \n",
       "\n",
       "   FAILED_JOB_NUMS   DOTTING_TIME RESOURCE_TYPE  DISK_USAGE  \n",
       "0                0  1590683100000            vm        20.0  \n",
       "1                0  1590683400000            vm        20.0  \n",
       "2                0  1590683700000            vm        20.0  \n",
       "3                0  1590684000000            vm        20.0  \n",
       "4                0  1590684120000            vm        20.0  \n",
       "5                0  1590684420000            vm        20.0  \n",
       "6                0  1590684720000            vm        20.0  \n",
       "7                0  1590685020000            vm        20.0  \n",
       "8                0  1590685320000            vm        20.0  \n",
       "9                0  1590685620000            vm        20.0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>QUEUE_ID</th>\n",
       "      <th>CU</th>\n",
       "      <th>STATUS</th>\n",
       "      <th>QUEUE_TYPE</th>\n",
       "      <th>PLATFORM</th>\n",
       "      <th>CPU_USAGE</th>\n",
       "      <th>MEM_USAGE</th>\n",
       "      <th>LAUNCHING_JOB_NUMS</th>\n",
       "      <th>RUNNING_JOB_NUMS</th>\n",
       "      <th>SUCCEED_JOB_NUMS</th>\n",
       "      <th>CANCELLED_JOB_NUMS</th>\n",
       "      <th>FAILED_JOB_NUMS</th>\n",
       "      <th>DOTTING_TIME</th>\n",
       "      <th>RESOURCE_TYPE</th>\n",
       "      <th>DISK_USAGE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>60</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1662213420000</td>\n",
       "      <td>vm</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>58</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1662213720000</td>\n",
       "      <td>vm</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>80</td>\n",
       "      <td>67</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1662214020000</td>\n",
       "      <td>vm</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>100</td>\n",
       "      <td>65</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1662214320000</td>\n",
       "      <td>vm</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>16</td>\n",
       "      <td>available</td>\n",
       "      <td>sql</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>98</td>\n",
       "      <td>67</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1662214620000</td>\n",
       "      <td>vm</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>85153</td>\n",
       "      <td>64</td>\n",
       "      <td>available</td>\n",
       "      <td>general</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>56</td>\n",
       "      <td>91</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1613655960000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>85153</td>\n",
       "      <td>64</td>\n",
       "      <td>available</td>\n",
       "      <td>general</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>48</td>\n",
       "      <td>78</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1613656260000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>85153</td>\n",
       "      <td>64</td>\n",
       "      <td>available</td>\n",
       "      <td>general</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>23</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1613656560000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>85153</td>\n",
       "      <td>64</td>\n",
       "      <td>available</td>\n",
       "      <td>general</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>68</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1613656860000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>85153</td>\n",
       "      <td>64</td>\n",
       "      <td>available</td>\n",
       "      <td>general</td>\n",
       "      <td>x86_64</td>\n",
       "      <td>38</td>\n",
       "      <td>74</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1613657160000</td>\n",
       "      <td>vm</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  QUEUE_ID  CU     STATUS QUEUE_TYPE PLATFORM  CPU_USAGE  MEM_USAGE  \\\n",
       "0   1       297  16  available        sql   x86_64         60         69   \n",
       "1   1       297  16  available        sql   x86_64         58         69   \n",
       "2   1       297  16  available        sql   x86_64         80         67   \n",
       "3   1       297  16  available        sql   x86_64        100         65   \n",
       "4   1       297  16  available        sql   x86_64         98         67   \n",
       "5   2     85153  64  available    general   x86_64         56         91   \n",
       "6   2     85153  64  available    general   x86_64         48         78   \n",
       "7   2     85153  64  available    general   x86_64         23         35   \n",
       "8   2     85153  64  available    general   x86_64         68         61   \n",
       "9   2     85153  64  available    general   x86_64         38         74   \n",
       "\n",
       "   LAUNCHING_JOB_NUMS  RUNNING_JOB_NUMS  SUCCEED_JOB_NUMS  CANCELLED_JOB_NUMS  \\\n",
       "0                   0                 5                 5                   0   \n",
       "1                   0                 9                 4                   0   \n",
       "2                   0                 9                 1                   0   \n",
       "3                   0                 7                 2                   0   \n",
       "4                   0                10                 3                   0   \n",
       "5                   0                 0                 0                   0   \n",
       "6                   0                 1                 1                   0   \n",
       "7                   0                 0                 0                   0   \n",
       "8                   0                 0                 0                   0   \n",
       "9                   0                 0                 0                   0   \n",
       "\n",
       "   FAILED_JOB_NUMS   DOTTING_TIME RESOURCE_TYPE  DISK_USAGE  \n",
       "0                0  1662213420000            vm           9  \n",
       "1                0  1662213720000            vm           9  \n",
       "2                0  1662214020000            vm           9  \n",
       "3                1  1662214320000            vm           9  \n",
       "4                1  1662214620000            vm           9  \n",
       "5                0  1613655960000            vm          20  \n",
       "6                0  1613656260000            vm          20  \n",
       "7                0  1613656560000            vm          20  \n",
       "8                0  1613656860000            vm          20  \n",
       "9                0  1613657160000            vm          20  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>CPU_USAGE_1</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_1</th>\n",
       "      <th>CPU_USAGE_2</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_2</th>\n",
       "      <th>CPU_USAGE_3</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_3</th>\n",
       "      <th>CPU_USAGE_4</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_4</th>\n",
       "      <th>CPU_USAGE_5</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  CPU_USAGE_1  LAUNCHING_JOB_NUMS_1  CPU_USAGE_2  LAUNCHING_JOB_NUMS_2  \\\n",
       "0   1            0                     0            0                     0   \n",
       "1   2            0                     0            0                     0   \n",
       "2   3            0                     0            0                     0   \n",
       "3   4            0                     0            0                     0   \n",
       "4   5            0                     0            0                     0   \n",
       "\n",
       "   CPU_USAGE_3  LAUNCHING_JOB_NUMS_3  CPU_USAGE_4  LAUNCHING_JOB_NUMS_4  \\\n",
       "0            0                     0            0                     0   \n",
       "1            0                     0            0                     0   \n",
       "2            0                     0            0                     0   \n",
       "3            0                     0            0                     0   \n",
       "4            0                     0            0                     0   \n",
       "\n",
       "   CPU_USAGE_5  LAUNCHING_JOB_NUMS_5  \n",
       "0            0                     0  \n",
       "1            0                     0  \n",
       "2            0                     0  \n",
       "3            0                     0  \n",
       "4            0                     0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_sample.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((501730, 15), (14980, 16), (2996, 11))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, test.shape, sub_sample.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 这些 columns 在 test 只有单一值, 所以直接去掉\n",
    "\n",
    "del train['STATUS']\n",
    "del train['PLATFORM']\n",
    "del train['RESOURCE_TYPE']\n",
    "\n",
    "del test['STATUS']\n",
    "del test['PLATFORM']\n",
    "del test['RESOURCE_TYPE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 时间排序好后也没什么用了\n",
    "\n",
    "del train['DOTTING_TIME']\n",
    "del test['DOTTING_TIME']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Label Encoding\n",
    "\n",
    "le = LabelEncoder()\n",
    "train['QUEUE_TYPE'] = le.fit_transform(train['QUEUE_TYPE'].astype(str))\n",
    "test['QUEUE_TYPE'] = le.transform(test['QUEUE_TYPE'].astype(str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1 CU = 1 CPU 4G MEM\n",
    "\n",
    "train['used_cpu'] = train['CU'] * train['CPU_USAGE'] / 100\n",
    "train['used_mem'] = train['CU'] * 4 * train['MEM_USAGE'] / 100\n",
    "\n",
    "test['used_cpu'] = test['CU'] * test['CPU_USAGE'] / 100\n",
    "test['used_mem'] = test['CU'] * 4 * test['MEM_USAGE'] / 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['to_run_jobs'] = train['LAUNCHING_JOB_NUMS'] - train['RUNNING_JOB_NUMS']\n",
    "test['to_run_jobs'] = test['LAUNCHING_JOB_NUMS'] - test['RUNNING_JOB_NUMS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# diffs\n",
    "\n",
    "train['used_cpu_diff1'] = train.groupby(['QUEUE_ID'])['used_cpu'].diff(1).fillna(0)\n",
    "train['used_mem_diff1'] = train.groupby(['QUEUE_ID'])['used_mem'].diff(1).fillna(0)\n",
    "train['used_disk_diff1'] = train.groupby(['QUEUE_ID'])['DISK_USAGE'].diff(1).fillna(0)\n",
    "train['to_run_jobs_diff1'] = train.groupby(['QUEUE_ID'])['to_run_jobs'].diff(1).fillna(0)\n",
    "train['launching_diff1'] = train.groupby(['QUEUE_ID'])['LAUNCHING_JOB_NUMS'].diff(1).fillna(0)\n",
    "train['running_diff1'] = train.groupby(['QUEUE_ID'])['RUNNING_JOB_NUMS'].diff(1).fillna(0)\n",
    "train['succeed_diff1'] = train.groupby(['QUEUE_ID'])['SUCCEED_JOB_NUMS'].diff(1).fillna(0)\n",
    "train['cancelled_diff1'] = train.groupby(['QUEUE_ID'])['CANCELLED_JOB_NUMS'].diff(1).fillna(0)\n",
    "train['failed_diff1'] = train.groupby(['QUEUE_ID'])['FAILED_JOB_NUMS'].diff(1).fillna(0)\n",
    "\n",
    "train['used_cpu_diff-1'] = train.groupby(['QUEUE_ID'])['used_cpu'].diff(-1).fillna(0)\n",
    "train['used_mem_diff-1'] = train.groupby(['QUEUE_ID'])['used_mem'].diff(-1).fillna(0)\n",
    "train['used_disk_diff-1'] = train.groupby(['QUEUE_ID'])['DISK_USAGE'].diff(-1).fillna(0)\n",
    "train['to_run_jobs_diff-1'] = train.groupby(['QUEUE_ID'])['to_run_jobs'].diff(-1).fillna(0)\n",
    "train['launching_diff-1'] = train.groupby(['QUEUE_ID'])['LAUNCHING_JOB_NUMS'].diff(-1).fillna(0)\n",
    "train['running_diff-1'] = train.groupby(['QUEUE_ID'])['RUNNING_JOB_NUMS'].diff(-1).fillna(0)\n",
    "train['succeed_diff-1'] = train.groupby(['QUEUE_ID'])['SUCCEED_JOB_NUMS'].diff(-1).fillna(0)\n",
    "train['cancelled_diff-1'] = train.groupby(['QUEUE_ID'])['CANCELLED_JOB_NUMS'].diff(-1).fillna(0)\n",
    "train['failed_diff-1'] = train.groupby(['QUEUE_ID'])['FAILED_JOB_NUMS'].diff(-1).fillna(0)\n",
    "\n",
    "\n",
    "test['used_cpu_diff1'] = test.groupby(['QUEUE_ID'])['used_cpu'].diff(1).fillna(0)\n",
    "test['used_mem_diff1'] = test.groupby(['QUEUE_ID'])['used_mem'].diff(1).fillna(0)\n",
    "test['used_disk_diff1'] = test.groupby(['QUEUE_ID'])['DISK_USAGE'].diff(1).fillna(0)\n",
    "test['to_run_jobs_diff1'] = test.groupby(['QUEUE_ID'])['to_run_jobs'].diff(1).fillna(0)\n",
    "test['launching_diff1'] = test.groupby(['QUEUE_ID'])['LAUNCHING_JOB_NUMS'].diff(1).fillna(0)\n",
    "test['running_diff1'] = test.groupby(['QUEUE_ID'])['RUNNING_JOB_NUMS'].diff(1).fillna(0)\n",
    "test['succeed_diff1'] = test.groupby(['QUEUE_ID'])['SUCCEED_JOB_NUMS'].diff(1).fillna(0)\n",
    "test['cancelled_diff1'] = test.groupby(['QUEUE_ID'])['CANCELLED_JOB_NUMS'].diff(1).fillna(0)\n",
    "test['failed_diff1'] = test.groupby(['QUEUE_ID'])['FAILED_JOB_NUMS'].diff(1).fillna(0)\n",
    "\n",
    "test['used_cpu_diff-1'] = test.groupby(['QUEUE_ID'])['used_cpu'].diff(-1).fillna(0)\n",
    "test['used_mem_diff-1'] = test.groupby(['QUEUE_ID'])['used_mem'].diff(-1).fillna(0)\n",
    "test['used_disk_diff-1'] = test.groupby(['QUEUE_ID'])['DISK_USAGE'].diff(-1).fillna(0)\n",
    "test['to_run_jobs_diff-1'] = test.groupby(['QUEUE_ID'])['to_run_jobs'].diff(-1).fillna(0)\n",
    "test['launching_diff-1'] = test.groupby(['QUEUE_ID'])['LAUNCHING_JOB_NUMS'].diff(-1).fillna(0)\n",
    "test['running_diff-1'] = test.groupby(['QUEUE_ID'])['RUNNING_JOB_NUMS'].diff(-1).fillna(0)\n",
    "test['succeed_diff-1'] = test.groupby(['QUEUE_ID'])['SUCCEED_JOB_NUMS'].diff(-1).fillna(0)\n",
    "test['cancelled_diff-1'] = test.groupby(['QUEUE_ID'])['CANCELLED_JOB_NUMS'].diff(-1).fillna(0)\n",
    "test['failed_diff-1'] = test.groupby(['QUEUE_ID'])['FAILED_JOB_NUMS'].diff(-1).fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>QUEUE_ID</th>\n",
       "      <th>CU</th>\n",
       "      <th>QUEUE_TYPE</th>\n",
       "      <th>CPU_USAGE</th>\n",
       "      <th>MEM_USAGE</th>\n",
       "      <th>LAUNCHING_JOB_NUMS</th>\n",
       "      <th>RUNNING_JOB_NUMS</th>\n",
       "      <th>SUCCEED_JOB_NUMS</th>\n",
       "      <th>CANCELLED_JOB_NUMS</th>\n",
       "      <th>FAILED_JOB_NUMS</th>\n",
       "      <th>DISK_USAGE</th>\n",
       "      <th>used_cpu</th>\n",
       "      <th>used_mem</th>\n",
       "      <th>to_run_jobs</th>\n",
       "      <th>used_cpu_diff1</th>\n",
       "      <th>used_mem_diff1</th>\n",
       "      <th>used_disk_diff1</th>\n",
       "      <th>to_run_jobs_diff1</th>\n",
       "      <th>launching_diff1</th>\n",
       "      <th>running_diff1</th>\n",
       "      <th>succeed_diff1</th>\n",
       "      <th>cancelled_diff1</th>\n",
       "      <th>failed_diff1</th>\n",
       "      <th>used_cpu_diff-1</th>\n",
       "      <th>used_mem_diff-1</th>\n",
       "      <th>used_disk_diff-1</th>\n",
       "      <th>to_run_jobs_diff-1</th>\n",
       "      <th>launching_diff-1</th>\n",
       "      <th>running_diff-1</th>\n",
       "      <th>succeed_diff-1</th>\n",
       "      <th>cancelled_diff-1</th>\n",
       "      <th>failed_diff-1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.48</td>\n",
       "      <td>34.56</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.32</td>\n",
       "      <td>34.56</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.80</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1.12</td>\n",
       "      <td>34.56</td>\n",
       "      <td>0</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.64</td>\n",
       "      <td>34.56</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.48</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.16</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.80</td>\n",
       "      <td>34.56</td>\n",
       "      <td>0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.32</td>\n",
       "      <td>-0.64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   QUEUE_ID  CU  QUEUE_TYPE  CPU_USAGE  MEM_USAGE  LAUNCHING_JOB_NUMS  \\\n",
       "0         2  16           2          3         54                   0   \n",
       "1         2  16           2          2         54                   0   \n",
       "2         2  16           2          7         54                   0   \n",
       "3         2  16           2          4         54                   0   \n",
       "4         2  16           2          5         54                   0   \n",
       "\n",
       "   RUNNING_JOB_NUMS  SUCCEED_JOB_NUMS  CANCELLED_JOB_NUMS  FAILED_JOB_NUMS  \\\n",
       "0                 0                 0                   0                0   \n",
       "1                 0                 0                   0                0   \n",
       "2                 0                 0                   0                0   \n",
       "3                 0                 0                   0                0   \n",
       "4                 0                 0                   0                0   \n",
       "\n",
       "   DISK_USAGE  used_cpu  used_mem  to_run_jobs  used_cpu_diff1  \\\n",
       "0        20.0      0.48     34.56            0            0.00   \n",
       "1        20.0      0.32     34.56            0           -0.16   \n",
       "2        20.0      1.12     34.56            0            0.80   \n",
       "3        20.0      0.64     34.56            0           -0.48   \n",
       "4        20.0      0.80     34.56            0            0.16   \n",
       "\n",
       "   used_mem_diff1  used_disk_diff1  to_run_jobs_diff1  launching_diff1  \\\n",
       "0             0.0              0.0                0.0              0.0   \n",
       "1             0.0              0.0                0.0              0.0   \n",
       "2             0.0              0.0                0.0              0.0   \n",
       "3             0.0              0.0                0.0              0.0   \n",
       "4             0.0              0.0                0.0              0.0   \n",
       "\n",
       "   running_diff1  succeed_diff1  cancelled_diff1  failed_diff1  \\\n",
       "0            0.0            0.0              0.0           0.0   \n",
       "1            0.0            0.0              0.0           0.0   \n",
       "2            0.0            0.0              0.0           0.0   \n",
       "3            0.0            0.0              0.0           0.0   \n",
       "4            0.0            0.0              0.0           0.0   \n",
       "\n",
       "   used_cpu_diff-1  used_mem_diff-1  used_disk_diff-1  to_run_jobs_diff-1  \\\n",
       "0             0.16             0.00               0.0                 0.0   \n",
       "1            -0.80             0.00               0.0                 0.0   \n",
       "2             0.48             0.00               0.0                 0.0   \n",
       "3            -0.16             0.00               0.0                 0.0   \n",
       "4             0.32            -0.64               0.0                 0.0   \n",
       "\n",
       "   launching_diff-1  running_diff-1  succeed_diff-1  cancelled_diff-1  \\\n",
       "0               0.0             0.0             0.0               0.0   \n",
       "1               0.0             0.0             0.0               0.0   \n",
       "2               0.0             0.0             0.0               0.0   \n",
       "3               0.0             0.0             0.0               0.0   \n",
       "4               0.0             0.0             0.0               0.0   \n",
       "\n",
       "   failed_diff-1  \n",
       "0            0.0  \n",
       "1            0.0  \n",
       "2            0.0  \n",
       "3            0.0  \n",
       "4            0.0  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加个 id 后面方便处理\n",
    "train['myid'] = train.index\n",
    "test['myid'] = test.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1300767c92854598a199131322257136",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=43.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 生成 target 列\n",
    "\n",
    "df_train = pd.DataFrame()\n",
    "\n",
    "for id_ in tqdm(train.QUEUE_ID.unique()):\n",
    "    tmp = train[train.QUEUE_ID == id_]\n",
    "    tmp['CPU_USAGE_next25mins'] = tmp['CPU_USAGE'].shift(-5)\n",
    "    tmp['LAUNCHING_JOB_NUMS_next25mins'] = tmp['LAUNCHING_JOB_NUMS'].shift(-5)\n",
    "    df_train = df_train.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(501515, 35)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>QUEUE_ID</th>\n",
       "      <th>CU</th>\n",
       "      <th>QUEUE_TYPE</th>\n",
       "      <th>CPU_USAGE</th>\n",
       "      <th>MEM_USAGE</th>\n",
       "      <th>LAUNCHING_JOB_NUMS</th>\n",
       "      <th>RUNNING_JOB_NUMS</th>\n",
       "      <th>SUCCEED_JOB_NUMS</th>\n",
       "      <th>CANCELLED_JOB_NUMS</th>\n",
       "      <th>FAILED_JOB_NUMS</th>\n",
       "      <th>DISK_USAGE</th>\n",
       "      <th>used_cpu</th>\n",
       "      <th>used_mem</th>\n",
       "      <th>to_run_jobs</th>\n",
       "      <th>used_cpu_diff1</th>\n",
       "      <th>used_mem_diff1</th>\n",
       "      <th>used_disk_diff1</th>\n",
       "      <th>to_run_jobs_diff1</th>\n",
       "      <th>launching_diff1</th>\n",
       "      <th>running_diff1</th>\n",
       "      <th>succeed_diff1</th>\n",
       "      <th>cancelled_diff1</th>\n",
       "      <th>failed_diff1</th>\n",
       "      <th>used_cpu_diff-1</th>\n",
       "      <th>used_mem_diff-1</th>\n",
       "      <th>used_disk_diff-1</th>\n",
       "      <th>to_run_jobs_diff-1</th>\n",
       "      <th>launching_diff-1</th>\n",
       "      <th>running_diff-1</th>\n",
       "      <th>succeed_diff-1</th>\n",
       "      <th>cancelled_diff-1</th>\n",
       "      <th>failed_diff-1</th>\n",
       "      <th>myid</th>\n",
       "      <th>CPU_USAGE_next25mins</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_next25mins</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.48</td>\n",
       "      <td>34.56</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.32</td>\n",
       "      <td>34.56</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.80</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1.12</td>\n",
       "      <td>34.56</td>\n",
       "      <td>0</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.64</td>\n",
       "      <td>34.56</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.48</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.16</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.80</td>\n",
       "      <td>34.56</td>\n",
       "      <td>0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.32</td>\n",
       "      <td>-0.64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   QUEUE_ID  CU  QUEUE_TYPE  CPU_USAGE  MEM_USAGE  LAUNCHING_JOB_NUMS  \\\n",
       "0         2  16           2          3         54                   0   \n",
       "1         2  16           2          2         54                   0   \n",
       "2         2  16           2          7         54                   0   \n",
       "3         2  16           2          4         54                   0   \n",
       "4         2  16           2          5         54                   0   \n",
       "\n",
       "   RUNNING_JOB_NUMS  SUCCEED_JOB_NUMS  CANCELLED_JOB_NUMS  FAILED_JOB_NUMS  \\\n",
       "0                 0                 0                   0                0   \n",
       "1                 0                 0                   0                0   \n",
       "2                 0                 0                   0                0   \n",
       "3                 0                 0                   0                0   \n",
       "4                 0                 0                   0                0   \n",
       "\n",
       "   DISK_USAGE  used_cpu  used_mem  to_run_jobs  used_cpu_diff1  \\\n",
       "0        20.0      0.48     34.56            0            0.00   \n",
       "1        20.0      0.32     34.56            0           -0.16   \n",
       "2        20.0      1.12     34.56            0            0.80   \n",
       "3        20.0      0.64     34.56            0           -0.48   \n",
       "4        20.0      0.80     34.56            0            0.16   \n",
       "\n",
       "   used_mem_diff1  used_disk_diff1  to_run_jobs_diff1  launching_diff1  \\\n",
       "0             0.0              0.0                0.0              0.0   \n",
       "1             0.0              0.0                0.0              0.0   \n",
       "2             0.0              0.0                0.0              0.0   \n",
       "3             0.0              0.0                0.0              0.0   \n",
       "4             0.0              0.0                0.0              0.0   \n",
       "\n",
       "   running_diff1  succeed_diff1  cancelled_diff1  failed_diff1  \\\n",
       "0            0.0            0.0              0.0           0.0   \n",
       "1            0.0            0.0              0.0           0.0   \n",
       "2            0.0            0.0              0.0           0.0   \n",
       "3            0.0            0.0              0.0           0.0   \n",
       "4            0.0            0.0              0.0           0.0   \n",
       "\n",
       "   used_cpu_diff-1  used_mem_diff-1  used_disk_diff-1  to_run_jobs_diff-1  \\\n",
       "0             0.16             0.00               0.0                 0.0   \n",
       "1            -0.80             0.00               0.0                 0.0   \n",
       "2             0.48             0.00               0.0                 0.0   \n",
       "3            -0.16             0.00               0.0                 0.0   \n",
       "4             0.32            -0.64               0.0                 0.0   \n",
       "\n",
       "   launching_diff-1  running_diff-1  succeed_diff-1  cancelled_diff-1  \\\n",
       "0               0.0             0.0             0.0               0.0   \n",
       "1               0.0             0.0             0.0               0.0   \n",
       "2               0.0             0.0             0.0               0.0   \n",
       "3               0.0             0.0             0.0               0.0   \n",
       "4               0.0             0.0             0.0               0.0   \n",
       "\n",
       "   failed_diff-1  myid  CPU_USAGE_next25mins  LAUNCHING_JOB_NUMS_next25mins  \n",
       "0            0.0     0                   3.0                            0.0  \n",
       "1            0.0     1                   2.0                            0.0  \n",
       "2            0.0     2                   2.0                            0.0  \n",
       "3            0.0     3                   5.0                            0.0  \n",
       "4            0.0     4                   6.0                            0.0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = df_train[df_train.CPU_USAGE_next25mins.notna()]\n",
    "# df_train['CPU_USAGE_next25mins'] /= 100\n",
    "\n",
    "print(df_train.shape)\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_lgb_qid(df_train, df_test, target, qid):\n",
    "    \n",
    "    feature_names = list(\n",
    "        filter(lambda x: x not in ['CPU_USAGE_next25mins', \n",
    "                                   'LAUNCHING_JOB_NUMS_next25mins', \n",
    "                                   'QUEUE_ID', \n",
    "                                   'myid',\n",
    "                                   'CU',\n",
    "                                   'QUEUE_TYPE'], df_train.columns))\n",
    "\n",
    "#     feature_names = ['CPU_USAGE', 'MEM_USAGE', 'LAUNCHING_JOB_NUMS',\n",
    "#                      'RUNNING_JOB_NUMS', 'SUCCEED_JOB_NUMS', 'CANCELLED_JOB_NUMS',\n",
    "#                      'FAILED_JOB_NUMS', 'DISK_USAGE', \n",
    "#                      'used_cpu', 'used_mem', \n",
    "#                      'used_cpu_diff1', 'used_mem_diff1',\n",
    "#                      'used_disk_diff1', 'to_run_jobs_diff1',\n",
    "#                      'launching_diff1', 'running_diff1',\n",
    "#                      'succeed_diff1', 'cancelled_diff1',\n",
    "#                      'failed_diff1',\n",
    "#                      'used_cpu_diff-1', 'used_mem_diff-1',\n",
    "#                      'used_disk_diff-1', 'to_run_jobs_diff-1',\n",
    "#                      'launching_diff-1', 'running_diff-1',\n",
    "#                      'succeed_diff-1', 'cancelled_diff-1',\n",
    "#                      'failed_diff-1']\n",
    "    \n",
    "    # 提取 QUEUE_ID 对应的数据集\n",
    "    df_train = df_train[df_train.QUEUE_ID == qid]\n",
    "    df_test = df_test[df_test.QUEUE_ID == qid]\n",
    "    \n",
    "    print(f\"QUEUE_ID:{qid}, target:{target}, train:{len(df_train)}, test:{len(df_test)}\")\n",
    "    \n",
    "    model = lgb.LGBMRegressor(num_leaves=32,\n",
    "                              max_depth=6,\n",
    "                              learning_rate=0.08,\n",
    "                              n_estimators=10000,\n",
    "                              subsample=0.9,\n",
    "                              feature_fraction=0.8,\n",
    "                              reg_alpha=0.5,\n",
    "                              reg_lambda=0.8,\n",
    "                              random_state=2020)\n",
    "    oof = []\n",
    "    prediction = df_test[['ID', 'QUEUE_ID', 'myid']]\n",
    "    prediction[target] = 0\n",
    "    \n",
    "    kfold = KFold(n_splits=5, random_state=2020)\n",
    "    for fold_id, (trn_idx, val_idx) in enumerate(kfold.split(df_train, df_train[target])):\n",
    "        \n",
    "        X_train = df_train.iloc[trn_idx][feature_names]\n",
    "        Y_train = df_train.iloc[trn_idx][target]\n",
    "        X_val = df_train.iloc[val_idx][feature_names]\n",
    "        Y_val = df_train.iloc[val_idx][target]\n",
    "        \n",
    "        lgb_model = model.fit(X_train, \n",
    "                              Y_train,\n",
    "                              eval_names=['train', 'valid'],\n",
    "                              eval_set=[(X_train, Y_train), (X_val, Y_val)],\n",
    "                              verbose=0,\n",
    "                              eval_metric='mse',\n",
    "                              early_stopping_rounds=20)\n",
    "        \n",
    "        pred_val = lgb_model.predict(X_val, num_iteration=lgb_model.best_iteration_)\n",
    "        df_oof = df_train.iloc[val_idx][[target, 'myid', 'QUEUE_ID']].copy()\n",
    "        df_oof['pred'] = pred_val\n",
    "        oof.append(df_oof)\n",
    "        \n",
    "        pred_test = lgb_model.predict(df_test[feature_names], num_iteration=lgb_model.best_iteration_)\n",
    "        prediction[target] += pred_test / kfold.n_splits\n",
    "        \n",
    "        del lgb_model, pred_val, pred_test, X_train, Y_train, X_val, Y_val\n",
    "        gc.collect()\n",
    "        \n",
    "    df_oof = pd.concat(oof)\n",
    "    score = mean_squared_error(df_oof[target], df_oof['pred'])\n",
    "    print('MSE:', score)\n",
    "\n",
    "    return prediction, score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89c7a10260be435fa244fa01df866ed0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "QUEUE_ID:297, target:CPU_USAGE_next25mins, train:21179, test:5710\n",
      "MSE: 190.33013812383643\n",
      "QUEUE_ID:297, target:LAUNCHING_JOB_NUMS_next25mins, train:21179, test:5710\n",
      "MSE: 6.439134989782293\n",
      "QUEUE_ID:85153, target:CPU_USAGE_next25mins, train:14348, test:1950\n",
      "MSE: 206.25600765498814\n",
      "QUEUE_ID:85153, target:LAUNCHING_JOB_NUMS_next25mins, train:14348, test:1950\n",
      "MSE: 14.774694229376149\n",
      "QUEUE_ID:291, target:CPU_USAGE_next25mins, train:8879, test:285\n",
      "MSE: 253.01064139163603\n",
      "QUEUE_ID:291, target:LAUNCHING_JOB_NUMS_next25mins, train:8879, test:285\n",
      "MSE: 0.04822364450600933\n",
      "QUEUE_ID:21487, target:CPU_USAGE_next25mins, train:28926, test:2235\n",
      "MSE: 78.21323592695981\n",
      "QUEUE_ID:21487, target:LAUNCHING_JOB_NUMS_next25mins, train:28926, test:2235\n",
      "MSE: 12.264486203235629\n",
      "QUEUE_ID:85265, target:CPU_USAGE_next25mins, train:13511, test:95\n",
      "MSE: 1.0642757707905275\n",
      "QUEUE_ID:85265, target:LAUNCHING_JOB_NUMS_next25mins, train:13511, test:95\n",
      "MSE: 30.074528990648865\n",
      "QUEUE_ID:4, target:CPU_USAGE_next25mins, train:19252, test:755\n",
      "MSE: 4.218780425737263\n",
      "QUEUE_ID:4, target:LAUNCHING_JOB_NUMS_next25mins, train:19252, test:755\n",
      "MSE: 0.27929391806304876\n",
      "QUEUE_ID:2, target:CPU_USAGE_next25mins, train:19250, test:755\n",
      "MSE: 6.219306555725542\n",
      "QUEUE_ID:2, target:LAUNCHING_JOB_NUMS_next25mins, train:19250, test:755\n",
      "MSE: 0.00020908211469530705\n",
      "QUEUE_ID:81221, target:CPU_USAGE_next25mins, train:19776, test:260\n",
      "MSE: 9.35160380648172\n",
      "QUEUE_ID:81221, target:LAUNCHING_JOB_NUMS_next25mins, train:19776, test:260\n",
      "MSE: 0.010564338173975327\n",
      "QUEUE_ID:287, target:CPU_USAGE_next25mins, train:6871, test:265\n",
      "MSE: 180.60806608193968\n",
      "QUEUE_ID:287, target:LAUNCHING_JOB_NUMS_next25mins, train:6871, test:265\n",
      "MSE: 0.3360009811750633\n",
      "QUEUE_ID:85693, target:CPU_USAGE_next25mins, train:10829, test:120\n",
      "MSE: 25.354944652542912\n",
      "QUEUE_ID:85693, target:LAUNCHING_JOB_NUMS_next25mins, train:10829, test:120\n",
      "MSE: 11.00993715184877\n",
      "QUEUE_ID:3, target:CPU_USAGE_next25mins, train:19252, test:780\n",
      "MSE: 3.974480939661742\n",
      "QUEUE_ID:3, target:LAUNCHING_JOB_NUMS_next25mins, train:19252, test:780\n",
      "MSE: 0.006028067642944448\n",
      "QUEUE_ID:293, target:CPU_USAGE_next25mins, train:8850, test:255\n",
      "MSE: 138.75052624956868\n",
      "QUEUE_ID:293, target:LAUNCHING_JOB_NUMS_next25mins, train:8850, test:255\n",
      "MSE: 0.13627534343007375\n",
      "QUEUE_ID:36, target:CPU_USAGE_next25mins, train:3237, test:165\n",
      "MSE: 135.84423412477165\n",
      "QUEUE_ID:36, target:LAUNCHING_JOB_NUMS_next25mins, train:3237, test:165\n",
      "MSE: 0.007395062022007127\n",
      "QUEUE_ID:26, target:CPU_USAGE_next25mins, train:10402, test:290\n",
      "MSE: 0.20002030519503392\n",
      "QUEUE_ID:26, target:LAUNCHING_JOB_NUMS_next25mins, train:10402, test:290\n",
      "MSE: 0.0\n",
      "QUEUE_ID:281, target:CPU_USAGE_next25mins, train:10360, test:500\n",
      "MSE: 0.5014339034905865\n",
      "QUEUE_ID:281, target:LAUNCHING_JOB_NUMS_next25mins, train:10360, test:500\n",
      "MSE: 0.00019305478471444572\n",
      "QUEUE_ID:83609, target:CPU_USAGE_next25mins, train:2115, test:15\n",
      "MSE: 68.32328455517326\n",
      "QUEUE_ID:83609, target:LAUNCHING_JOB_NUMS_next25mins, train:2115, test:15\n",
      "MSE: 21.353244176862482\n",
      "QUEUE_ID:21671, target:CPU_USAGE_next25mins, train:28090, test:145\n",
      "MSE: 4.604060345964039\n",
      "QUEUE_ID:21671, target:LAUNCHING_JOB_NUMS_next25mins, train:28090, test:145\n",
      "MSE: 6.872018821478845\n",
      "QUEUE_ID:27, target:CPU_USAGE_next25mins, train:10521, test:285\n",
      "MSE: 0.869908165854854\n",
      "QUEUE_ID:27, target:LAUNCHING_JOB_NUMS_next25mins, train:10521, test:285\n",
      "MSE: 9.50540171236403e-05\n",
      "QUEUE_ID:233, target:CPU_USAGE_next25mins, train:2094, test:55\n",
      "MSE: 133.3291001041898\n",
      "QUEUE_ID:233, target:LAUNCHING_JOB_NUMS_next25mins, train:2094, test:55\n",
      "MSE: 0.0\n",
      "QUEUE_ID:85101, target:CPU_USAGE_next25mins, train:6613, test:10\n",
      "MSE: 0.8279715261893207\n",
      "QUEUE_ID:85101, target:LAUNCHING_JOB_NUMS_next25mins, train:6613, test:10\n",
      "MSE: 21.477346714553285\n",
      "QUEUE_ID:85933, target:CPU_USAGE_next25mins, train:8806, test:20\n",
      "MSE: 38.625897778362955\n",
      "QUEUE_ID:85933, target:LAUNCHING_JOB_NUMS_next25mins, train:8806, test:20\n",
      "MSE: 4.068381404132899\n",
      "QUEUE_ID:21673, target:CPU_USAGE_next25mins, train:19721, test:20\n",
      "MSE: 3.557371191887047\n",
      "QUEUE_ID:21673, target:LAUNCHING_JOB_NUMS_next25mins, train:19721, test:20\n",
      "MSE: 0.0\n",
      "QUEUE_ID:298, target:CPU_USAGE_next25mins, train:20376, test:10\n",
      "MSE: 8.103905272704441\n",
      "QUEUE_ID:298, target:LAUNCHING_JOB_NUMS_next25mins, train:20376, test:10\n",
      "MSE: 0.9798969917822608\n",
      "\n"
     ]
    }
   ],
   "source": [
    "oofs1 = list()\n",
    "oofs2 = list()\n",
    "predictions1 = list()\n",
    "predictions2 = list()\n",
    "scores1 = list()\n",
    "scores2 = list()\n",
    "\n",
    "for qid in tqdm(test.QUEUE_ID.unique()):\n",
    "    prediction1, score1 = run_lgb_qid(df_train, test, target='CPU_USAGE_next25mins', qid=qid)\n",
    "    predictions1.append(prediction1)\n",
    "    scores1.append(score1)\n",
    "    prediction2, score2 = run_lgb_qid(df_train, test, target='LAUNCHING_JOB_NUMS_next25mins', qid=qid)\n",
    "    predictions2.append(prediction2)\n",
    "    scores2.append(score2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64.87561716755006 5.6581716617230935\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(scores1), np.mean(scores2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions1 = pd.concat(predictions1)\n",
    "predictions2 = pd.concat(predictions2)\n",
    "\n",
    "predictions1 = predictions1.sort_values(by='myid').reset_index(drop=True)\n",
    "predictions2 = predictions2.sort_values(by='myid').reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>QUEUE_ID</th>\n",
       "      <th>myid</th>\n",
       "      <th>CPU_USAGE_next25mins</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_next25mins</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>0</td>\n",
       "      <td>11.343928</td>\n",
       "      <td>0.021107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>1</td>\n",
       "      <td>12.731280</td>\n",
       "      <td>0.024004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>2</td>\n",
       "      <td>19.294139</td>\n",
       "      <td>0.025891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>3</td>\n",
       "      <td>95.529173</td>\n",
       "      <td>0.702751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>4</td>\n",
       "      <td>83.432941</td>\n",
       "      <td>0.081439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14975</td>\n",
       "      <td>2996</td>\n",
       "      <td>287</td>\n",
       "      <td>14975</td>\n",
       "      <td>2.840110</td>\n",
       "      <td>0.029591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14976</td>\n",
       "      <td>2996</td>\n",
       "      <td>287</td>\n",
       "      <td>14976</td>\n",
       "      <td>5.747585</td>\n",
       "      <td>0.023989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14977</td>\n",
       "      <td>2996</td>\n",
       "      <td>287</td>\n",
       "      <td>14977</td>\n",
       "      <td>5.009150</td>\n",
       "      <td>0.026029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14978</td>\n",
       "      <td>2996</td>\n",
       "      <td>287</td>\n",
       "      <td>14978</td>\n",
       "      <td>4.434103</td>\n",
       "      <td>0.023251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14979</td>\n",
       "      <td>2996</td>\n",
       "      <td>287</td>\n",
       "      <td>14979</td>\n",
       "      <td>3.983136</td>\n",
       "      <td>0.023325</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14980 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         ID  QUEUE_ID   myid  CPU_USAGE_next25mins  \\\n",
       "0         1       297      0             11.343928   \n",
       "1         1       297      1             12.731280   \n",
       "2         1       297      2             19.294139   \n",
       "3         1       297      3             95.529173   \n",
       "4         1       297      4             83.432941   \n",
       "...     ...       ...    ...                   ...   \n",
       "14975  2996       287  14975              2.840110   \n",
       "14976  2996       287  14976              5.747585   \n",
       "14977  2996       287  14977              5.009150   \n",
       "14978  2996       287  14978              4.434103   \n",
       "14979  2996       287  14979              3.983136   \n",
       "\n",
       "       LAUNCHING_JOB_NUMS_next25mins  \n",
       "0                           0.021107  \n",
       "1                           0.024004  \n",
       "2                           0.025891  \n",
       "3                           0.702751  \n",
       "4                           0.081439  \n",
       "...                              ...  \n",
       "14975                       0.029591  \n",
       "14976                       0.023989  \n",
       "14977                       0.026029  \n",
       "14978                       0.023251  \n",
       "14979                       0.023325  \n",
       "\n",
       "[14980 rows x 5 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction = predictions1.copy()\n",
    "prediction = pd.merge(prediction, predictions2[['myid', 'LAUNCHING_JOB_NUMS_next25mins']], on='myid')\n",
    "\n",
    "prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    14980.000000\n",
       "mean        16.036306\n",
       "std         15.114320\n",
       "min         -9.143809\n",
       "25%          4.448658\n",
       "50%         11.680019\n",
       "75%         21.347691\n",
       "max         99.670620\n",
       "Name: CPU_USAGE_next25mins, dtype: float64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction.CPU_USAGE_next25mins.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    14980.000000\n",
       "mean         0.545399\n",
       "std          3.285629\n",
       "min         -0.290879\n",
       "25%          0.013331\n",
       "50%          0.021885\n",
       "75%          0.223851\n",
       "max         52.382402\n",
       "Name: LAUNCHING_JOB_NUMS_next25mins, dtype: float64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction.LAUNCHING_JOB_NUMS_next25mins.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>CPU_USAGE_1</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_1</th>\n",
       "      <th>CPU_USAGE_2</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_2</th>\n",
       "      <th>CPU_USAGE_3</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_3</th>\n",
       "      <th>CPU_USAGE_4</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_4</th>\n",
       "      <th>CPU_USAGE_5</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  CPU_USAGE_1  LAUNCHING_JOB_NUMS_1  CPU_USAGE_2  LAUNCHING_JOB_NUMS_2  \\\n",
       "0   1            0                     0            0                     0   \n",
       "1   2            0                     0            0                     0   \n",
       "2   3            0                     0            0                     0   \n",
       "3   4            0                     0            0                     0   \n",
       "4   5            0                     0            0                     0   \n",
       "\n",
       "   CPU_USAGE_3  LAUNCHING_JOB_NUMS_3  CPU_USAGE_4  LAUNCHING_JOB_NUMS_4  \\\n",
       "0            0                     0            0                     0   \n",
       "1            0                     0            0                     0   \n",
       "2            0                     0            0                     0   \n",
       "3            0                     0            0                     0   \n",
       "4            0                     0            0                     0   \n",
       "\n",
       "   CPU_USAGE_5  LAUNCHING_JOB_NUMS_5  \n",
       "0            0                     0  \n",
       "1            0                     0  \n",
       "2            0                     0  \n",
       "3            0                     0  \n",
       "4            0                     0  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_sample.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>QUEUE_ID</th>\n",
       "      <th>myid</th>\n",
       "      <th>CPU_USAGE_next25mins</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_next25mins</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>3</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>297</td>\n",
       "      <td>4</td>\n",
       "      <td>83</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14975</td>\n",
       "      <td>2996</td>\n",
       "      <td>287</td>\n",
       "      <td>14975</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14976</td>\n",
       "      <td>2996</td>\n",
       "      <td>287</td>\n",
       "      <td>14976</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14977</td>\n",
       "      <td>2996</td>\n",
       "      <td>287</td>\n",
       "      <td>14977</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14978</td>\n",
       "      <td>2996</td>\n",
       "      <td>287</td>\n",
       "      <td>14978</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14979</td>\n",
       "      <td>2996</td>\n",
       "      <td>287</td>\n",
       "      <td>14979</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14980 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         ID  QUEUE_ID   myid  CPU_USAGE_next25mins  \\\n",
       "0         1       297      0                    11   \n",
       "1         1       297      1                    12   \n",
       "2         1       297      2                    19   \n",
       "3         1       297      3                    95   \n",
       "4         1       297      4                    83   \n",
       "...     ...       ...    ...                   ...   \n",
       "14975  2996       287  14975                     2   \n",
       "14976  2996       287  14976                     5   \n",
       "14977  2996       287  14977                     5   \n",
       "14978  2996       287  14978                     4   \n",
       "14979  2996       287  14979                     3   \n",
       "\n",
       "       LAUNCHING_JOB_NUMS_next25mins  \n",
       "0                                  0  \n",
       "1                                  0  \n",
       "2                                  0  \n",
       "3                                  0  \n",
       "4                                  0  \n",
       "...                              ...  \n",
       "14975                              0  \n",
       "14976                              0  \n",
       "14977                              0  \n",
       "14978                              0  \n",
       "14979                              0  \n",
       "\n",
       "[14980 rows x 5 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 注意: 提交要求预测结果需为非负整数\n",
    "\n",
    "prediction['CPU_USAGE_next25mins'] = prediction['CPU_USAGE_next25mins'].apply(np.floor)\n",
    "prediction['CPU_USAGE_next25mins'] = prediction['CPU_USAGE_next25mins'].apply(lambda x: 0 if x<0 else x)\n",
    "prediction['CPU_USAGE_next25mins'] = prediction['CPU_USAGE_next25mins'].astype(int)\n",
    "prediction['LAUNCHING_JOB_NUMS_next25mins'] = prediction['LAUNCHING_JOB_NUMS_next25mins'].apply(np.floor)\n",
    "prediction['LAUNCHING_JOB_NUMS_next25mins'] = prediction['LAUNCHING_JOB_NUMS_next25mins'].apply(lambda x: 0 if x<0 else x)\n",
    "prediction['LAUNCHING_JOB_NUMS_next25mins'] = prediction['LAUNCHING_JOB_NUMS_next25mins'].astype(int)\n",
    "\n",
    "prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4cefae6b4e74680a21683e37ae5f0e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2996.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "preds = []\n",
    "\n",
    "for id_ in tqdm(prediction.ID.unique()):\n",
    "    items = [id_]\n",
    "    tmp = prediction[prediction.ID == id_].sort_values(by='myid').reset_index(drop=True)\n",
    "    for i, row in tmp.iterrows():\n",
    "        items.append(row['CPU_USAGE_next25mins'])\n",
    "        items.append(row['LAUNCHING_JOB_NUMS_next25mins'])\n",
    "    preds.append(items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>CPU_USAGE_1</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_1</th>\n",
       "      <th>CPU_USAGE_2</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_2</th>\n",
       "      <th>CPU_USAGE_3</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_3</th>\n",
       "      <th>CPU_USAGE_4</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_4</th>\n",
       "      <th>CPU_USAGE_5</th>\n",
       "      <th>LAUNCHING_JOB_NUMS_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "      <td>83</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  CPU_USAGE_1  LAUNCHING_JOB_NUMS_1  CPU_USAGE_2  LAUNCHING_JOB_NUMS_2  \\\n",
       "0   1           11                     0           12                     0   \n",
       "1   2           18                     0           18                     0   \n",
       "2   3           17                     0           34                     0   \n",
       "3   4           13                     0           20                     0   \n",
       "4   5            5                     0           11                     0   \n",
       "5   6            8                     0           12                     0   \n",
       "6   7           10                     0            8                     0   \n",
       "7   8            0                    25            1                    31   \n",
       "8   9            3                     0            3                     0   \n",
       "9  10           14                     0           14                     0   \n",
       "\n",
       "   CPU_USAGE_3  LAUNCHING_JOB_NUMS_3  CPU_USAGE_4  LAUNCHING_JOB_NUMS_4  \\\n",
       "0           19                     0           95                     0   \n",
       "1           30                     0           38                     0   \n",
       "2           10                     0           13                     0   \n",
       "3            6                     0           11                     0   \n",
       "4           10                     0           12                     0   \n",
       "5            8                     0           10                     0   \n",
       "6            8                     0           12                     0   \n",
       "7            0                    32            0                    32   \n",
       "8            3                     0            3                     0   \n",
       "9           12                     0            8                     0   \n",
       "\n",
       "   CPU_USAGE_5  LAUNCHING_JOB_NUMS_5  \n",
       "0           83                     0  \n",
       "1           21                     0  \n",
       "2           12                     0  \n",
       "3           16                     0  \n",
       "4           21                     0  \n",
       "5           13                     0  \n",
       "6           20                     0  \n",
       "7            0                    14  \n",
       "8            3                     0  \n",
       "9           10                     0  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub = pd.DataFrame(preds)\n",
    "sub.columns = sub_sample.columns\n",
    "\n",
    "sub.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2996, 11), (2996, 11))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.shape, sub_sample.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv('baseline_202010151337.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
